#coding:utf-8
import os
import requests
import re, json
from urllib.parse import urlencode
'''
    作用：抓取今日头条街拍搜索结果，并下载子页面内图片
    注：“街拍”为关键词，可以替换其他关键词进行搜索
'''


def imgSpider(keyword, beginPage, endPage):
    '''
        作用:今日头条街拍图片获取，获取搜索的页面数据，提取其中URL
    '''
    for page in range(beginPage, endPage):
        if not keyword:
            keyword = '街拍'

        params = {
            'offset':page*20,
            'format':'json',
            'keyword':keyword,
            'autoload':'true',
            'count': '20',
            'cur_tab':'3',
            'from':'gallery'
        }
        url = 'https://www.toutiao.com/search_content/?' + urlencode(params)

        try:
            response = requests.get(url)
            if response.status_code == 200:
                items = response.json().get('data')
                for item in items:
                    url = item.get('article_url')
                    if url:
                        loadPage(url)
        except requests.ConnectionError as e:
            print('Error',e.args)

def loadPage(url):
    '''
        作用：爬取相应页面
        url:爬取的地址
    '''
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    try:

        html = requests.get(url, headers=headers).text

        #使用正则提取title文本，作为文件夹名称
        folderName = re.search(r'<title>(.*?)</title>', html)
        if folderName:
            folderName = folderName.group(1)
            print(folderName)
        #使用正则提取图片链接
        patten = re.compile(r'sub_images.*?:(.*?),\\"max', re.S)
        result = re.search(patten, html)
        #如果匹配不到，会返回none
        if  result:
            result = result.group(1)

            result = result.encode('utf-8').decode('unicode_escape')
            items = re.sub(r'\\','',result)
            items = json.loads(items)
            for item in items:
                url = item.get('url')
                #下载图片
                loadImg(url, folderName)
        

    except requests.ConnectionError as e:
        print('Error',e.args)

def loadImg(src,folderName):
    '''
        根据图片地址，请求图片并存储到本地
    '''
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    image = requests.get(src, headers=headers).content
    imgName = src.split('/')[-1]
    #判断并创建文件夹
    path = './pic/'+ folderName
    isExists = os.path.exists(path)
    if not isExists:
        os.makedirs(path)

    with open(path + '/' + imgName, 'wb') as f:
        f.write(image)

if __name__ == "__main__":
    keyword = str(input("请输入关键词："))
    beginPage = int(input("请输入起始页："))
    endPage = int(input("请输入结束页："))

    imgSpider(keyword, beginPage, endPage)